package srcBlackJack;

import java.io.BufferedWriter;
import java.io.FileWriter;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;

import lib.ArgumentValidator;
import lib.logging.ANSIConsoleLogger;
import lib.logging.ILogger;

public class BlackJackStrategy 
{
    private static int ComputeBestScore(int minSum, boolean hasAce) 
    {
        if (hasAce && (minSum + 10) <= 21)
        {
            return (minSum + 10);
        }

        return minSum;
    }

    private static int ComputeReward(
            BlackJackState srcState,
            BlackJackState destState,
            GameStatus gameStatus)
    {
        if (gameStatus == GameStatus.PlayerWon)
        {
            return 200;
        }
        
        if (gameStatus == GameStatus.PlayerWonWithBlackJack)
        {
            return 250;
        }
        if (gameStatus == GameStatus.DealerWon)
        {
            return -100;
        }
        
        if (gameStatus == GameStatus.Tie)
        {
            return 100;
        }
        
        int pBestScore1 = ComputeBestScore(srcState.pMinSum, srcState.pHasAce);
        int dBestScore1 = ComputeBestScore(srcState.dMinSum, srcState.dHasAce);
        
        int pBestScore2 = ComputeBestScore(destState.pMinSum, destState.pHasAce);
        int dBestScore2 = ComputeBestScore(destState.dMinSum, destState.dHasAce);
        
        return ((pBestScore2 - dBestScore2) - (pBestScore1 - dBestScore1));
    }

    private static char GetCharForAction(Action action) 
    {
        switch (action)
        {
            case Double:
                return 'D';
            case Hit:
                return 'H';
            case None:
                return 'N';
            case Split:
                return 'P';
            case Stand:
                return 'S';
        }
        
        // unreachable statement
        return 0;
    }

    private static void InsertIntoTransitionInfos(List<TransitionInfo> tInfos, TransitionInfo tInfo)
    {
        int index;
        if ((index = tInfos.indexOf(tInfo)) != -1)
        {
            double newProbability = tInfo.Probability() + tInfos.get(index).Probability();
            tInfos.get(index).SetProbability(newProbability);
        }
        else
        {
            tInfos.add(tInfo);
        }
    }
    
    public static void main(String[] args) throws Exception 
    {
        ILogger logger = new ANSIConsoleLogger();
        
        try
        {
            if (args.length != 1)
            {
                logger.LogError(
                        "Incorrect number of paramets: %d. " +
                        "Usage: java BlackJackStrategy <faceCardProbability>",
                        args.length);
    
                System.exit(1);
            }

            double fcProbability = Double.parseDouble(args[0]);
            BlackJackStrategy bjs = new BlackJackStrategy(fcProbability, logger);
            bjs.GenerateStateMap();
            
            System.out.println("numStates: " + bjs.stateMap.keySet().size());
            System.out.println("numTransitionEntries: " + bjs.transitionMap.keySet().size());
            
            OutputStrategy(bjs.stateMap, "Policy.txt");
        }
        catch (Exception ex)
        {
            logger.LogException(ex);
            throw ex;
        }
    }
    
    private static void OutputStrategy(HashMap<BlackJackState, StateInfo> stateMap, String outputFile) throws IOException 
    {
        BufferedWriter bufWriter = new BufferedWriter(
                new FileWriter(outputFile));
        
        // print no ace no pair
        for (int i = 5; i <= 19; ++i)
        {
            bufWriter.write(Integer.toString(i));
            bufWriter.write('\t');
            
            for (int j = 2; j <= 10; ++j)
            {
                BlackJackState state = new BlackJackState(i, j, 0, false, false, false, false, true, false, true, false, j == 10);
                Action action = stateMap.get(state).OptimalAction();
                
                bufWriter.write(GetCharForAction(action));
                bufWriter.write(' ');
            }
            
            // output dealer Ace possibility
            BlackJackState dealerAceState = new BlackJackState(i, 1, 0, false, true, false, false, true, false, true, true, false);
            Action action = stateMap.get(dealerAceState).OptimalAction();
            bufWriter.write(GetCharForAction(action));
            
            bufWriter.newLine();
        }
        
        // print one Ace
        for (int i = 2; i <= 9; ++i)
        {
            bufWriter.write('A');
            bufWriter.write(Integer.toString(i));
            bufWriter.write('\t');
            
            for (int j = 2; j <= 10; ++j)
            {
                BlackJackState state = new BlackJackState(i + 1, j, 0, true, false, false, false, true, false, true, false, j == 10);
                Action action = stateMap.get(state).OptimalAction();
                
                bufWriter.write(GetCharForAction(action));
                bufWriter.write(' ');
            }
            
            // output dealer Ace possibility
            BlackJackState dealerAceState = new BlackJackState(i + 1, 1, 0, true, true, false, false, true, false, true, true, false);
            Action action = stateMap.get(dealerAceState).OptimalAction();
            bufWriter.write(GetCharForAction(action));
            
            bufWriter.newLine();
        }
        
        // print pairs
        for (int i = 2; i <= 10; ++i)
        {
            bufWriter.write(Integer.toString(i));
            bufWriter.write(Integer.toString(i));
            bufWriter.write('\t');
            
            for (int j = 2; j <= 10; ++j)
            {
                BlackJackState state = new BlackJackState(2 * i, j, i, false, false, false, false, true, false, true, false, j == 10);
                Action action = stateMap.get(state).OptimalAction();
                
                bufWriter.write(GetCharForAction(action));
                bufWriter.write(' ');
            }
            
            // output dealer Ace possibility
            BlackJackState dealerAceState = new BlackJackState(2 * i, 1, i, false, true, false, false, true, false, true, true, false);
            Action action = stateMap.get(dealerAceState).OptimalAction();
            bufWriter.write(GetCharForAction(action));
            
            bufWriter.newLine();
        }
        
        // print ace pair
        bufWriter.write('A');
        bufWriter.write('A');
        bufWriter.write('\t');
        
        for (int j = 2; j <= 10; ++j)
        {
            BlackJackState state = new BlackJackState(2, j, 1, true, false, false, false, true, false, true, false, j == 10);
            Action action = stateMap.get(state).OptimalAction();
            
            bufWriter.write(GetCharForAction(action));
            bufWriter.write(' ');
        }
        
        // output dealer Ace possibility
        BlackJackState dealerAceState = new BlackJackState(2, 1, 1, true, true, false, false, true, false, true, true, false);
        Action action = stateMap.get(dealerAceState).OptimalAction();
        bufWriter.write(GetCharForAction(action));
        
        bufWriter.close();
    }
    
    private HashMap<BlackJackState, StateInfo> stateMap;
    private HashMap<BlackJackState, List<TransitionInfo>> transitionMap;
    private ILogger logger;
    
    private double faceCardProbability;
    private double oneNonFaceCardProbability;
    
    private static final double DiscountFactor = 1.0;
    private static final double Epsilon = 0.000001;
    private static final int MaxIterations = 200;
    
    public BlackJackStrategy(double faceCardProbability, ILogger logger)
    {
        // for game to be well formed, p < 0.5
        ArgumentValidator.IsGreaterThanOrEqualTo("faceCardProbability", faceCardProbability, 0.0);
        ArgumentValidator.IsLessThan("faceCardProbability", faceCardProbability, 0.5);
        ArgumentValidator.IsNotNull("logger", logger);
        
        this.faceCardProbability = faceCardProbability;
        this.oneNonFaceCardProbability = (1 - faceCardProbability) / 9;
        this.stateMap = new HashMap<BlackJackState, StateInfo>();
        this.transitionMap = new HashMap<BlackJackState, List<TransitionInfo>>();
        this.logger = logger;
    }

    private int ComputeDealerMinSumValueForTerminalState(int pBestScore, int dBestScore) 
    {
        if (dBestScore > pBestScore)
        {
            return 3;
        }
        else
        {
            return 2;
        }
    }

    private int ComputePlayerMinSumValueForTerminalState(int pBestScore, int dBestScore) 
    {
        if (pBestScore > dBestScore)
        {
            return 3;
        }
        else
        {
            return 2;
        }
    }
    
    private HashMap<Action, List<ActionMapElem>> GenerateActionMap(List<TransitionInfo> tInfos) 
    {
        HashMap<Action, List<ActionMapElem>> map = new HashMap<Action, List<ActionMapElem>>();
        
        for (TransitionInfo tInfo : tInfos)
        {
            if (map.containsKey(tInfo.ApplicableAction()))
            {
                map.get(tInfo.ApplicableAction()).add(
                        new ActionMapElem(
                                tInfo.DestinationState(),
                                tInfo.Probability(),
                                tInfo.Reward()));
            }
            else
            {
                List<ActionMapElem> elems = new ArrayList<ActionMapElem>();
                elems.add(new ActionMapElem(tInfo.DestinationState(), tInfo.Probability(), tInfo.Reward()));
                map.put(tInfo.ApplicableAction(), elems);
            }
        }
        
        return map;
    }
    
    private void GenerateDealerBustedStates() 
    {
        // Collapsing a parameter means, if it's value doesn't matter in a similar set of states,
        // choose one particular value for the parameter, and just create one state instead.
        //
        // collapse values of pHasAce to false
        // collapse values of dHasAce to false
        // collapse values of dMinSum (>= 22) to 22
        // collapse values of pMinSum (>= 2 ) to 23 to maximize reward
        // collapse values of pairValue (>= 0) to 0
        // collapse values of pHasTwoCards to false
        BlackJackState state = new BlackJackState(23, 22, 0, false, false, false, false, false, true, false, false, false);

        StateInfo sInfo = new StateInfo(Action.None, 2.0, 0);
        
        this.stateMap.put(state, sInfo);
    }

    private void GenerateDealerWinsByBlackJackStates() 
    {
        // Collapsing a parameter means, if it's value doesn't matter in a similar set of states,
        // choose one particular value for the parameter, and just create one state instead.
        //
        // collapse values of pHasAce to false
        // collapse values of pMinSum (>= 2 ) to 2 to minimize reward
        // collapse pairValue (>= 0) to 0
        // collapse values of pairValue (>= 0) to 0
        // collapse values of pHasTwoCards to false
        BlackJackState state = new BlackJackState(2, 11, 0, false, true, false, true, false, true, false, false, false);

        StateInfo sInfo = new StateInfo(Action.None, 0.0, 0);
        
        this.stateMap.put(state, sInfo);
    }

    private void GenerateDealerWonWithBetterHandStates() 
    {
        // Collapsing a parameter means, if it's value doesn't matter in a similar set of states,
        // choose one particular value for the parameter, and just create one state instead.
        //
        // collapse values of pHasAce to false
        // collapse values of dHasAce to false
        // pMinSum > dMinSum, collapse all such values to 2 > 3
        // collapse values of pairValue (>= 0) to 0
        // collapse values of pHasTwoCards to false
        BlackJackState state = new BlackJackState(2, 3, 0, false, false, false, false, false, true, false, false, false);

        StateInfo sInfo = new StateInfo(Action.None, 0.0, 0);
        
        this.stateMap.put(state, sInfo);
    }

    /**
     * Generates possible states when player has a pair in initial hand
     * 
     */
    private void GenerateInditialStatesWithPairs() 
    {
        for (int playerCard = 1; playerCard <= 10; ++playerCard)
        {
            for (int dealerCard = 1; dealerCard <= 10; ++dealerCard)
            {
                BlackJackState state = new BlackJackState(
                        2 * playerCard,
                        dealerCard,
                        playerCard,
                        playerCard == 1,
                        dealerCard == 1,
                        false,
                        false,
                        true,//false,
                        false,
                        true,
                        dealerCard == 1,
                        dealerCard == 10);
                
                StateInfo sInfo = new StateInfo(Action.None, 0, Double.MAX_VALUE);
                
                this.stateMap.put(state, sInfo);
            }
        }
    }

    private void GenerateInitialSetOfStates() 
    {
        GenerateInitialStatesWithNoAceNorPairs();
        GenerateInitialStatesWithOneAce();
        GenerateInditialStatesWithPairs();
    }

    /**
     * Generates possible states when player does not have a Ace nor a pair initial hand
     * 
     */
    private void GenerateInitialStatesWithNoAceNorPairs() 
    {
        for (int playerSum = 5; playerSum <= 19; ++playerSum)
        {
            for (int dealerCard = 1; dealerCard <= 10; ++dealerCard)
            {
                BlackJackState state = new BlackJackState(
                        playerSum,
                        dealerCard,
                        0,
                        false,
                        dealerCard == 1,
                        false,
                        false,
                        true,//false,
                        false,
                        true,
                        dealerCard == 1,
                        dealerCard == 10);
                
                StateInfo sInfo = new StateInfo(Action.None, 0, Double.MAX_VALUE);
                
                this.stateMap.put(state, sInfo);
            }
        }
    }

    /**
     * Generates possible states when player has exactly one Ace in initial hand
     * 
     */
    private void GenerateInitialStatesWithOneAce() 
    {
        for (int playerCard = 2; playerCard <= 9; ++playerCard)
        {
            for (int dealerCard = 1; dealerCard <= 10; ++dealerCard)
            {
                BlackJackState state = new BlackJackState(
                        playerCard + 1, // since it's minSum, value of Ace would be counted as 1
                        dealerCard,
                        0,
                        true,
                        dealerCard == 1,
                        false,
                        false,
                        true,//false,
                        false,
                        true,
                        dealerCard == 1,
                        dealerCard == 10);
                
                StateInfo sInfo = new StateInfo(Action.None, 0, Double.MAX_VALUE);
                
                this.stateMap.put(state, sInfo);
            }
        }
    }

    private void GeneratePlayerBlackJackStates() 
    {
        // Collapsing a parameter means, if it's value doesn't matter in a similar set of states,
        // choose one particular value for the parameter, and just create one state instead.
        //
        // collapse values of dHasBlackJack to false
        // collapse values of dHasAce to false
        // collapse values of dMinSum (>= 2) to 2
        // collapse values of dHasOnlyAceCard to false
        // collapse values of dHasOnlyFaceCard to false
        BlackJackState state = new BlackJackState(11, 2, 0, true, false, true, false, false, true, true, false, false);

        StateInfo sInfo = new StateInfo(Action.None, 2.5, 0);
        
        this.stateMap.put(state, sInfo);
    }

    private void GeneratePlayerBustedStates() 
    {
        // Collapsing a parameter means, if it's value doesn't matter in a similar set of states,
        // choose one particular value for the parameter, and just create one state instead.
        //
        // collapse values of dHasBlackJack to false
        // collapse values of dHasAce to false
        // collapse values of dMinSum (>= 2) to 23 to minimize reward for player
        // collapse values of pMinSum (>= 22) to 22
        // collapse values of dHasOnlyAceCard to false
        // collapse values of dHasOnlyFaceCard to false
        BlackJackState state = new BlackJackState(22, 23, 0, false, false, false, false, false, true, false, false, false);

        StateInfo sInfo = new StateInfo(Action.None, 0.0, 0);
        
        this.stateMap.put(state, sInfo);
    }

    private void GeneratePlayerWonWithBetterHandStates() 
    {
        // Collapsing a parameter means, if it's value doesn't matter in a similar set of states,
        // choose one particular value for the parameter, and just create one state instead.
        //
        // collapse values of pHasAce to false
        // collapse values of dHasAce to false
        // pMinSum > dMinSum, collapse all such values to 3 > 2
        // collapse values of pairValue (>= 0) to 0
        // collapse values of pHasTwoCards to false
        BlackJackState state = new BlackJackState(3, 2, 0, false, false, false, false, false, true, false, false, false);

        StateInfo sInfo = new StateInfo(Action.None, 2.0, 0);
        
        this.stateMap.put(state, sInfo);
    }

    private void GeneratePushStates() 
    {
        // Collapsing a parameter means, if it's value doesn't matter in a similar set of states,
        // choose one particular value for the parameter, and just create one state instead.
        //
        // collapse values of pHasAce to false
        // collapse values of dHasAce to false
        // pMinSum == dMinSum, collapse all such values to pMinSum = dMinSum = 2
        // collapse values of pairValue (>= 0) to 0
        // collapse values of pHasTwoCards to false
        BlackJackState state = new BlackJackState(2, 2, 0, false, false, false, false, false, true, false, false, false);

        StateInfo sInfo = new StateInfo(Action.None, 1.0, 0);
        
        this.stateMap.put(state, sInfo);
    }

    public void GenerateStateMap()
    {
        GenerateTerminalStates();
        GenerateInitialSetOfStates();
        
        int counter = 0;
        
        while (counter < MaxIterations)
        {
            counter++;
            
            double maxDelta = PerformValueIterationOverStates();
            
            logger.LogInfo("Iteration %d: maxDelta: %f", counter, maxDelta);
            
            if (maxDelta < Epsilon)
            {
                logger.LogInfo(
                        "Solution converged! Iteration %d: maxDelta: %f epsilon: %f",
                        counter,
                        maxDelta,
                        Epsilon);
                break;
            }
        }
    }

    private void GenerateTerminalStates() 
    {
        GeneratePlayerBlackJackStates();
        GeneratePlayerBustedStates();
        GenerateDealerBustedStates();
        GenerateDealerWonWithBetterHandStates();
        GeneratePlayerWonWithBetterHandStates();
        GenerateDealerWinsByBlackJackStates();
        GeneratePushStates();
    }

    private List<TransitionInfo> GenerateTransitionInfos(BlackJackState state) 
    {
        if (state.IsTerminalState())
        {
            return new ArrayList<TransitionInfo>();
        }
        
        if (state.PTurn())
        {
            return GenerateTransitionInfosForPlayer(state);
        }

        return GenerateTransitionInfosForDealer(state);
    }

    private List<TransitionInfo> GenerateTransitionInfosForDealer(BlackJackState state) 
    {
        List<TransitionInfo> tInfos = new ArrayList<TransitionInfo>();
        
        for (int i = 1; i < 10; ++i)
        {
            InsertIntoTransitionInfos(tInfos, GetDealerTransitionInfo(i, oneNonFaceCardProbability, state));
        }
        InsertIntoTransitionInfos(tInfos, GetDealerTransitionInfo(10, faceCardProbability, state));
        
        return tInfos;
    }

    private List<TransitionInfo> GenerateTransitionInfosForPlayer(BlackJackState state) 
    {
        List<TransitionInfo> tInfos = new ArrayList<TransitionInfo>();
        
        // Generate transitions due to Hit
        for (int i = 1; i < 10; ++i)
        {
            InsertIntoTransitionInfos(tInfos, GetPlayerHitTransitionInfo(i, oneNonFaceCardProbability, state));
        }
        InsertIntoTransitionInfos(tInfos, GetPlayerHitTransitionInfo(10, faceCardProbability, state));

        // Generate transitions due to Double-down
        if (state.PHasTwoCards())
        {
            for (int i = 1; i < 10; ++i)
            {
                InsertIntoTransitionInfos(tInfos, GetPlayerDoubleDownTransitionInfo(i, oneNonFaceCardProbability, state));
            }
            InsertIntoTransitionInfos(tInfos, GetPlayerDoubleDownTransitionInfo(10, faceCardProbability, state));
        }
        
        // Generate transitions due to Stay
        InsertIntoTransitionInfos(tInfos, GetPlayerStayTransitionInfo(state));
        
        return tInfos;
    }

    private TransitionInfo GetDealerTransitionInfo(
            int cardValue,
            double probability,
            BlackJackState state)
    {
        Action action = Action.Hit;
        
        // check for dealer blackjack
        if ((state.DHasOnlyAceCard() && cardValue == 10)
                || state.dHasOnlyFaceCard && cardValue == 1)
        {
            BlackJackState destinationState = new BlackJackState(
                    2,
                    11,
                    0,
                    false,
                    true,
                    false,
                    true,
                    false,
                    true,
                    false,
                    false,
                    false);
            
            int reward = ComputeReward(state, destinationState, GameStatus.DealerWon);
            return new TransitionInfo(action, destinationState, probability, reward);
        }
        
        int baseScore = state.DMinSum() + cardValue;
        
     // check for dealer busting
        if (baseScore > 21)
        {
            BlackJackState dealerBustedState = new BlackJackState(
                    23,
                    22,
                    0,
                    false,
                    false,
                    false,
                    false,
                    false,
                    true,
                    false,
                    false,
                    false);
            
            int reward = ComputeReward(state, dealerBustedState, GameStatus.PlayerWon);
            return new TransitionInfo(action, dealerBustedState, probability, reward);
        }
        
        int scoreWithAceAdjustment = state.DHasAce() || cardValue == 1 ? (baseScore + 10) : baseScore;
        
        // check for dealer hitting soft 17
        
        if ((baseScore >= 17 && baseScore <= 21)
                || scoreWithAceAdjustment >= 17 && scoreWithAceAdjustment <= 21)
        {
            
            int pBestScore = ComputeBestScore(state.pMinSum, state.pHasAce);
            int dBestScore = scoreWithAceAdjustment <= 21 ? scoreWithAceAdjustment : baseScore;
            
            int destStatePMinSum = ComputePlayerMinSumValueForTerminalState(pBestScore, dBestScore);
            int destStateDMinSum = ComputeDealerMinSumValueForTerminalState(pBestScore, dBestScore);
            
            BlackJackState destinationState = new BlackJackState(
                    destStatePMinSum,
                    destStateDMinSum,
                    0,
                    false,
                    false,
                    false,
                    false,
                    false,
                    true,
                    false,
                    false,
                    false);
            
            GameStatus gameStatus;
            
            if (destStatePMinSum == destStateDMinSum)
            {
                gameStatus = GameStatus.Tie;
            }
            else if (destStatePMinSum > destStateDMinSum)
            {
                gameStatus = GameStatus.PlayerWon;
            }
            else
            {
                gameStatus = GameStatus.DealerWon;
            }
            
            int reward = ComputeReward(state, destinationState, gameStatus);
            return new TransitionInfo(action, destinationState, probability, reward);
        }
        
        // control reaching here means, we have a non-terminal state
        BlackJackState destinationState = new BlackJackState(
                state.PMinSum(),
                baseScore,
                state.PairValue(),
                state.PHasAce(),
                state.DHasAce() || cardValue == 1,
                false,
                false,
                false,
                false,
                state.PHasTwoCards(),
                false,
                false);
        
        int reward = ComputeReward(state, destinationState, GameStatus.InProgress);
        return new TransitionInfo(action, destinationState, probability, reward);
    }

    private double GetMaxDeltaInState() 
    {
        double maxDelta = 0.0;
        int statesWithInfiniteDelta = 0;
        
        for (BlackJackState state : stateMap.keySet())
        {
            double delta = stateMap.get(state).DeltaFromPrevIteration();
            
            if (delta == Double.MAX_VALUE)
            {
                ++statesWithInfiniteDelta;
            }
            
            if (delta > maxDelta)
            {
                maxDelta = delta;
            }
        }
        
        logger.LogInfo(
                "total states: %d, statesWithInfiniteDelta: %d",
                stateMap.keySet().size(),
                statesWithInfiniteDelta);
        
        return maxDelta;
    }

    private TransitionInfo GetPlayerDoubleDownTransitionInfo(
            int cardValue,
            double probability,
            BlackJackState state) 
    {
        Action action = Action.Double;
        
        int baseScore = state.PMinSum() + cardValue;
        
        // check for player busting
        if (baseScore > 21)
        {
            BlackJackState playerBustedState = new BlackJackState(
                    22,
                    23,
                    0,
                    false,
                    false,
                    false,
                    false,
                    false,
                    true,
                    false,
                    false,
                    false);
            
            int reward = 2 * ComputeReward(state, playerBustedState, GameStatus.DealerWon);
            return new TransitionInfo(action, playerBustedState, probability, reward);
        }
        
        // control reaching here means, we have a non-terminal state
        BlackJackState destinationState = new BlackJackState(
                baseScore,
                state.DMinSum(),
                0,
                state.PHasAce() || cardValue == 1,
                state.DHasAce(),
                false,
                false,
                false,
                false,
                false,
                state.DHasOnlyAceCard(),
                state.DHasOnlyFaceCard());
                
        int reward = 2 * ComputeReward(state, destinationState, GameStatus.InProgress);
        return new TransitionInfo(action, destinationState, probability, reward);
    }

    private TransitionInfo GetPlayerHitTransitionInfo(
            int cardValue,
            double probability,
            BlackJackState state) 
    {
        Action action = Action.Hit;
        
        int baseScore = state.PMinSum() + cardValue;
        
        // check for player busting
        if (baseScore > 21)
        {
            BlackJackState playerBustedState = new BlackJackState(
                    22,
                    23,
                    0,
                    false,
                    false,
                    false,
                    false,
                    false,
                    true,
                    false,
                    false,
                    false);
            
            int reward = ComputeReward(state, playerBustedState, GameStatus.DealerWon);
            return new TransitionInfo(action, playerBustedState, probability, reward);
        }
        
        // control reaching here means, we have a non-terminal state
        BlackJackState destinationState = new BlackJackState(
                baseScore,
                state.DMinSum(),
                0,
                state.PHasAce() || cardValue == 1,
                state.DHasAce(),
                false,
                false,
                true,
                false,
                false,
                state.DHasOnlyAceCard(),
                state.DHasOnlyFaceCard());
                
        int reward = ComputeReward(state, destinationState, GameStatus.InProgress);
        return new TransitionInfo(action, destinationState, probability, reward);
    }
    
    private TransitionInfo GetPlayerStayTransitionInfo(BlackJackState state) 
    {
        BlackJackState nextState = new BlackJackState(
                state.PMinSum(),
                state.DMinSum(),
                0,
                state.PHasAce(),
                state.DHasAce(),
                false,
                false,
                false,
                false,
                state.PHasTwoCards(),
                state.DHasOnlyAceCard(),
                state.DHasOnlyFaceCard());
        
        int reward = ComputeReward(state, nextState, GameStatus.InProgress);
        return new TransitionInfo(Action.Stand, nextState, 1.0, reward);
    }

    private void GetPossibleStatesOnSplitting(
            BlackJackState splittableState,
            HashMap<BlackJackState, Double> splitStatesWithProbabilites) 
    {
        ArgumentValidator.IsGreaterThan("pairValue", splittableState.PairValue(), 0);
        
        int firstCardValue = splittableState.PairValue();
        
        // handle Ace split separately
        if (firstCardValue == 1)
        {
            for (int i = 1; i <= 10; ++i)
            {
                double probability = i == 10 ? faceCardProbability : oneNonFaceCardProbability;
                
                BlackJackState state = new BlackJackState(
                        firstCardValue + i,
                        splittableState.DMinSum(),
                        i == firstCardValue ? i : 0,
                        (firstCardValue == 1) || (i == 1),
                        splittableState.DHasAce(),
                        false,
                        false,
                        false,
                        false,
                        true,
                        splittableState.DHasOnlyAceCard(),
                        splittableState.DHasOnlyFaceCard());
                
                splitStatesWithProbabilites.put(state, probability);
            }
        }
        else
        {
            for (int i = 1; i <= 10; ++i)
            {
                double probability = i == 10 ? faceCardProbability : oneNonFaceCardProbability;
                
                if (firstCardValue == 10 && i == 1)
                {
                    BlackJackState playerBackJackState = new BlackJackState(
                            11,
                            2,
                            0,
                            true,
                            false,
                            true,
                            false,
                            false,
                            true,
                            true,
                            false,
                            false);
                    
                    splitStatesWithProbabilites.put(playerBackJackState, probability);
                }
                else
                {
                    BlackJackState state = new BlackJackState(
                            firstCardValue + i,
                            splittableState.DMinSum(),
                            i == firstCardValue ? i : 0,
                            (firstCardValue == 1) || (i == 1),
                            splittableState.DHasAce(),
                            false,
                            false,
                            true,
                            false,
                            true,
                            splittableState.DHasOnlyAceCard(),
                            splittableState.DHasOnlyFaceCard());
                    
                    splitStatesWithProbabilites.put(state, probability);
                }
            }
        }
    }

    private StateInfo GetStateInfo(
            BlackJackState nborState,
            HashMap<BlackJackState,StateInfo> additionalStatesMap) 
    {
        if (!stateMap.containsKey(nborState)
                && !additionalStatesMap.containsKey(nborState))
        {
            if (nborState.IsTerminalState())
            {
                throw new IllegalStateException(
                        String.format(
                                "StateMap does not contain an entry for a terminal state: %s",
                                nborState));
            }
            
            Action action = Action.Hit;
            double expectedReward = 0.0;
            double delta = Double.MAX_VALUE;
            additionalStatesMap.put(nborState, new StateInfo(action, expectedReward, delta));
        }
        
        return stateMap.containsKey(nborState) ? stateMap.get(nborState) : additionalStatesMap.get(nborState);
    }

    private List<TransitionInfo> GetTransitionInfos(BlackJackState state) 
    {
        if (this.transitionMap.containsKey(state))
        {
            return this.transitionMap.get(state);
        }
        
        List<TransitionInfo> transitionInfos = GenerateTransitionInfos(state);
        this.transitionMap.put(state, transitionInfos);

        return transitionInfos;
    }

    private boolean IsActionApplicable(
            Action action,
            BlackJackState state,
            HashMap<Action, List<ActionMapElem>> actionMap) 
    {
        switch (action)
        {
            case Hit:
                return actionMap.containsKey(Action.Hit);
            case Stand:
                return actionMap.containsKey(Action.Stand);
            case Double:
                return actionMap.containsKey(Action.Double);
            case Split:
                return state.PHasTwoCards() && (state.PairValue() > 0);
            default:
                    return false;
        }
    }

    private void MergeAdditionalStatesMap(HashMap<BlackJackState, StateInfo> additionalStatesMap) 
    {
        for (BlackJackState additionalState : additionalStatesMap.keySet())
        {
            if (stateMap.containsKey(additionalState))
            {
                throw new IllegalStateException(
                        String.format(
                                "StateMap already contains entry for state: %s",
                                additionalState));
            }
            
            stateMap.put(additionalState, additionalStatesMap.get(additionalState));
        }
    }

    private double PerformValueIterationOverStates() 
    {
        HashMap<BlackJackState, StateInfo> additionalStatesMap = new HashMap<BlackJackState, StateInfo>();
        
        for (BlackJackState state : this.stateMap.keySet())
        {
            List<TransitionInfo> transitionInfos = GetTransitionInfos(state);
            
            HashMap<Action, List<ActionMapElem>> actionMap = GenerateActionMap(transitionInfos);
            
            boolean isAnyActionApplicable = false;
            
            double hitCaseReward = Double.MIN_VALUE;
            double standCaseReward = Double.MIN_VALUE;
            double doubleDownCaseReward = Double.MIN_VALUE;
            double splitCaseReward = Double.MIN_VALUE;
            
            if (IsActionApplicable(Action.Hit, state, actionMap))
            {
                isAnyActionApplicable = true;
                
                for (ActionMapElem amElem : actionMap.get(Action.Hit))
                {
                    BlackJackState nborState = amElem.NextState();
                    StateInfo sInfo = GetStateInfo(nborState, additionalStatesMap);
                    double expectedReward = amElem.Probability() * (amElem.Reward() + DiscountFactor * sInfo.OptimalExpectedReward());
                    hitCaseReward += expectedReward;
                }
            }
            
            if (IsActionApplicable(Action.Stand, state, actionMap))
            {
                isAnyActionApplicable = true;
                
                for (ActionMapElem amElem : actionMap.get(Action.Stand))
                {
                    BlackJackState nborState = amElem.NextState();
                    StateInfo sInfo = GetStateInfo(nborState, additionalStatesMap);
                    double expectedReward = amElem.Probability() * (amElem.Reward() + DiscountFactor * sInfo.OptimalExpectedReward());
                    standCaseReward += expectedReward;
                }
            }
            
            
            if (IsActionApplicable(Action.Double, state, actionMap))
            {
                isAnyActionApplicable = true;
                
                for (ActionMapElem amElem : actionMap.get(Action.Double))
                {
                    BlackJackState nborState = amElem.NextState();
                    StateInfo sInfo = GetStateInfo(nborState, additionalStatesMap);
                    double expectedReward = amElem.Probability() * (amElem.Reward() + DiscountFactor * sInfo.OptimalExpectedReward());
                    doubleDownCaseReward += expectedReward;
                }
            }
            
            
            if (IsActionApplicable(Action.Split, state, actionMap))
            {
                isAnyActionApplicable = true;
                
                HashMap<BlackJackState, Double> splitStatesWithProbabilites = new HashMap<BlackJackState, Double>();
                
                GetPossibleStatesOnSplitting(state, splitStatesWithProbabilites);
                
                 for (BlackJackState splitState : splitStatesWithProbabilites.keySet())
                 {
                     StateInfo sInfo = GetStateInfo(splitState, additionalStatesMap);
                     double probability = splitStatesWithProbabilites.get(splitState);
                     double expectedReward = probability * sInfo.OptimalExpectedReward();
                     splitCaseReward += expectedReward;
                 }
                 
                 // since we play two hands simultaneously, we double the expected reward
                 splitCaseReward *= 2;
            }
            
            
            if (isAnyActionApplicable)
            {
                double maxExpectedReward = hitCaseReward;
                Action bestAction = Action.Hit;
                
                if (standCaseReward > maxExpectedReward)
                {
                    maxExpectedReward = standCaseReward;
                    bestAction = Action.Stand;
                }
                
                if (doubleDownCaseReward > maxExpectedReward)
                {
                    maxExpectedReward = doubleDownCaseReward;
                    bestAction = Action.Double;
                }
                
                if (splitCaseReward > maxExpectedReward)
                {
                    maxExpectedReward = splitCaseReward;
                    bestAction = Action.Split;
                }
                
                UpdateStateInStatesMap(state, bestAction, maxExpectedReward, additionalStatesMap);
            }
        }
        
        MergeAdditionalStatesMap(additionalStatesMap);

        return GetMaxDeltaInState();
    }

    private void UpdateStateInStatesMap(
            BlackJackState state,
            Action bestAction,
            double maxExpectedReward,
            HashMap<BlackJackState, StateInfo> additionalStatesMap) 
    {
        ArgumentValidator.IsEqualTo("isTerminalState", state.IsTerminalState(), false);
        
        StateInfo sInfo;
        
        if (stateMap.containsKey(state))
        {
            sInfo = stateMap.get(state);
        }
        else if (additionalStatesMap.containsKey(state))
        {
            sInfo = additionalStatesMap.get(state);
        }
        else
        {
            throw new IllegalStateException(
                    String.format(
                            "StateMap or AdditionalStatesMap does not contain an entry for state: %s",
                            state));
        }

        double delta = Math.abs(sInfo.OptimalExpectedReward() - maxExpectedReward);
        sInfo.SetOptimalAction(bestAction);
        sInfo.SetOptimalExpectedReward(maxExpectedReward);
        sInfo.SetDeltaFromPrevIteration(delta);
    }
}
